package domain;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.Charset;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import com.mongodb.BasicDBObject;
import com.mongodb.DBCollection;
import com.mongodb.DBConnector;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;

public class Bing {
	public Bing(String ip)
	{
		String content = "";
		String name_domain;
		try {

			int page=0;
			DBCollection coll = Data.data("domainjp");
//			coll.createIndex(new BasicDBObject("name", -1));
			while(1==1)
			{
				content = "";
				URL dataURL = new URL("http://www.bing.com/search?q=site:"+ip+"&go=&qs=n&sk=&first="+(page*10+1)+"&FORM=PERE");
				
				page++;
				
				HttpURLConnection connection = (HttpURLConnection) dataURL.openConnection();
				connection.setReadTimeout(500000);
				connection.setConnectTimeout(1000000);
				BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), Charset.forName("UTF8")));
				while(1==1)
				{
					String str = reader.readLine();
					if(str==null) break;
					content+=str;
					
				}
				
				Document doc = Jsoup.parse(content);
				
				try {
					for(int i=0;i<doc.select("cite").size();i++)
					{
						name_domain = doc.select("cite").get(i).text().replaceAll("[ ]+", "").replaceAll("www.", "").replaceAll("[\\/]+[\\w\\W]+", "");
						System.out.println(name_domain);
						DBObject db = coll.findOne(new BasicDBObject("name",name_domain));
						if(db==null)
						{
							BasicDBObject ins = new BasicDBObject();
							ins.put("name", name_domain);
							coll.insert(ins);
							System.out.println(name_domain);
						}
					}
				} catch (Exception e) {
					// TODO: handle exception
				}
				if(doc.select("a[class=sb_pagN]").size() <= 0)
				{
					break;
				}
			}
		} catch (Exception e) {
			
			e.printStackTrace();
		}

	}
	public static void main( String args[])
	{
		DBCollection coll = Data.data("noun");
//		coll.createIndex(new BasicDBObject("name", -1));
		DBCursor db = coll.find();
		if(db.size()>0)
		{
			while(db.hasNext())
			{
				new Bing("jp+"+db.next().get("name").toString().replaceAll("[ ]+", "+"));
			}
		}
	}
}
